import settings 
import os
import gzip
import re
import time
import sys
import util
import logging
from threading import Thread
from settings import log

#class Extract(Thread):
    #def __init__(self):
        #Thread.__init__(self)
        
        #self.status = -1
    
    #def run(self):
        ## extract file from .gz 
        #try:
            #contents = gzip.open(settings.dirInput + hostfile ,'rb').read()
            #print '>>> Extract ' + hostfile
        #except IOError :
            #print 'ERROR> ' + hostfile + ' is not a gzip format.'
        
        ## ExtractSites 
        #page_count,url_count = util.ExtractSites(contents)
        #page_all += page_count
        #url_all += url_count
        #print '\t',len(contents),'bytes\t',url_count,'urls\t',page_count,'pages\t',time.time()-start_t,'s'

def main():
    # open connection
    page_all = 0
    url_all = 0

    # List file in DataTest (hostname == filename)
    for hostid , hostfile in zip(range(settings.maximumHost),os.listdir(settings.dirInput)):
        start_t = time.time()
        # extract file from .gz 
        try:
            contents = gzip.open(settings.dirInput + hostfile ,'rb').read()
            log.info('>>> Extract ' + hostfile)
            
            # ExtractSites 
            page_count,url_count = util.ExtractSites(contents)
            page_all += page_count
            url_all += url_count
            log.info('\t'+str(len(contents))+'bytes\t'+str(url_count)+'urls\t'+str(page_count)+'pages\t'+str(time.time()-start_t)+'s')
        except IOError :
            log.info('ERROR> ' + hostfile + ' is not a gzip format.')

    log.info('......................END...........................\n')
    log.info(str(hostid)+'hosts'+str(url_all)+'urls'+str(page_all)+'pages')


if __name__ == "__main__" :
    start_t = time.time()
    main()
    log.info(time.ctime())
    print 'Overall time usage:',time.time()-start_t,'s\n'
    log.info(time.ctime())
    os.system('rm *.pyc')
